{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import re\n",
    "import os\n",
    "import nltk\n",
    "import pandas as pd\n",
    "from keras.utils import np_utils\n",
    "\n",
    "\n",
    "from keras.layers import Dense ,LSTM,concatenate,Input,Flatten\n",
    "from keras import backend as K\n",
    "from keras.utils.data_utils import get_file\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Input, Dense, Activation, merge, Flatten, Reshape\n",
    "from keras.layers import LSTM, Bidirectional\n",
    "from keras.models import Model\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras import optimizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Problem Forumlation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model is provided two inputs question_vector and document_vector and asked to predict the start and end index of answer in the document"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_path = \"./GloVe/glove.840B.300d.txt\"\n",
    "embedding_path = \"/Users/rajatjain/Documents/new_webapp/react-app-kubernetes/backend/src/app/data/glove/glove.6B.100d.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_json('./data/train-v1.1.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = []\n",
    "questions = []\n",
    "answers = []\n",
    "titles = []\n",
    "answer_start_indexs = []\n",
    "answer_end_indexs = []\n",
    "def get_attributes(item):\n",
    "    data = item['data']\n",
    "    title = data['title']\n",
    "    for paragraph in data['paragraphs']:\n",
    "        for qas in paragraph['qas']:\n",
    "            answer = qas['answers'][0]['text']\n",
    "            answer_start_index = qas['answers'][0]['answer_start']\n",
    "            answer_end_index = answer_start_index + len(answer.split(' ')) - 1\n",
    "            answers.append(qas['answers'][0]['text'])\n",
    "            questions.append(qas['question'])\n",
    "            documents.append(paragraph['context'])\n",
    "            answer_start_indexs.append(answer_start_index)\n",
    "            answer_end_indexs.append(answer_end_index)\n",
    "            \n",
    "            titles.append(title)\n",
    "            \n",
    "def build_dataframe(train):\n",
    "    train.apply(get_attributes, axis = 1)\n",
    "    train_df = pd.DataFrame({\n",
    "    'documents':documents,\n",
    "    'questions': questions,\n",
    "    'answers': answers,\n",
    "    'titles': titles,\n",
    "     'answer_end_indexs': answer_end_indexs,\n",
    "    'answer_start_indexs': answer_start_indexs\n",
    "})\n",
    "    return train_df\n",
    "    \n",
    "train_df = build_dataframe(train)\n",
    "train_df = train_df.head(5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>documents</th>\n",
       "      <th>questions</th>\n",
       "      <th>answers</th>\n",
       "      <th>titles</th>\n",
       "      <th>answer_end_indexs</th>\n",
       "      <th>answer_start_indexs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>To whom did the Virgin Mary allegedly appear i...</td>\n",
       "      <td>Saint Bernadette Soubirous</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>517</td>\n",
       "      <td>515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What is in front of the Notre Dame Main Building?</td>\n",
       "      <td>a copper statue of Christ</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>192</td>\n",
       "      <td>188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>The Basilica of the Sacred heart at Notre Dame...</td>\n",
       "      <td>the Main Building</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>281</td>\n",
       "      <td>279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What is the Grotto at Notre Dame?</td>\n",
       "      <td>a Marian place of prayer and reflection</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>387</td>\n",
       "      <td>381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What sits on top of the Main Building at Notre...</td>\n",
       "      <td>a golden statue of the Virgin Mary</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>98</td>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           documents  \\\n",
       "0  Architecturally, the school has a Catholic cha...   \n",
       "1  Architecturally, the school has a Catholic cha...   \n",
       "2  Architecturally, the school has a Catholic cha...   \n",
       "3  Architecturally, the school has a Catholic cha...   \n",
       "4  Architecturally, the school has a Catholic cha...   \n",
       "\n",
       "                                           questions  \\\n",
       "0  To whom did the Virgin Mary allegedly appear i...   \n",
       "1  What is in front of the Notre Dame Main Building?   \n",
       "2  The Basilica of the Sacred heart at Notre Dame...   \n",
       "3                  What is the Grotto at Notre Dame?   \n",
       "4  What sits on top of the Main Building at Notre...   \n",
       "\n",
       "                                   answers                    titles  \\\n",
       "0               Saint Bernadette Soubirous  University_of_Notre_Dame   \n",
       "1                a copper statue of Christ  University_of_Notre_Dame   \n",
       "2                        the Main Building  University_of_Notre_Dame   \n",
       "3  a Marian place of prayer and reflection  University_of_Notre_Dame   \n",
       "4       a golden statue of the Virgin Mary  University_of_Notre_Dame   \n",
       "\n",
       "   answer_end_indexs  answer_start_indexs  \n",
       "0                517                  515  \n",
       "1                192                  188  \n",
       "2                281                  279  \n",
       "3                387                  381  \n",
       "4                 98                   92  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_max_length(sentences):\n",
    "    max_length = 0\n",
    "    for sentence in sentences:\n",
    "        length_of_sentence = len(sentence)\n",
    "        if length_of_sentence > max_length:\n",
    "            max_length = length_of_sentence\n",
    "    return max_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = train_df.head(2000)\n",
    "documents = list(train_df[\"documents\"])\n",
    "questions = list(train_df[\"questions\"])\n",
    "answer_start_indexs = train_df[\"answer_start_indexs\"].values\n",
    "answer_end_indexs = train_df[\"answer_end_indexs\"].values\n",
    "sentences = documents + questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = train_df['questions'].values\n",
    "answers = train_df['answers'].values\n",
    "documents = train_df['documents'].values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorize Question, Answer and Context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorized_data = []\n",
    "def vectorize(item):\n",
    "    tokenizer = Tokenizer(\n",
    "    num_words = 20000,\n",
    "    filters = '\"#$%&()*+-/:;<=>@[\\]^_`{|}~'   \n",
    ")\n",
    "        \n",
    "    documents = list(item[\"documents\"])\n",
    "    questions = list(item[\"questions\"])\n",
    "    answers = list(item['answers'])\n",
    "    start_index = list(item['answer_start_indexs'])\n",
    "    end_index = list(item['answer_end_indexs'])\n",
    "    sentences = documents + questions\n",
    "    \n",
    "    tokenizer.fit_on_texts(sentences)\n",
    "    questions_tokenized = tokenizer.texts_to_sequences(questions)\n",
    "    answers_tokenized = tokenizer.texts_to_sequences(answers)\n",
    "    documents_tokenized = tokenizer.texts_to_sequences(documents)\n",
    "    \n",
    "    questions_padded = pad_sequences(questions_tokenized, maxlen = 80, padding = 'post')\n",
    "    answers_padded = pad_sequences(answers_tokenized, maxlen = 1405, padding = 'post')\n",
    "    documents_padded = pad_sequences(documents_tokenized, maxlen = 1405, padding = 'post')\n",
    "    for i in range(0, len(documents)):\n",
    "        vectorized_data.append([questions_padded[i], answers_padded[i], documents_padded[i], start_index[i], end_index[i] ])\n",
    "    \n",
    "train_df.groupby('documents').apply(vectorize)\n",
    "vectorized_data = pd.DataFrame(vectorized_data)\n",
    "vectorized_data.rename(columns = {0: 'question_vector', 1: 'answers_vector', 2: 'documents_vector', 3: 'answer_start_indexs', 4: 'answer_end_indexs' },inplace = True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question_vector</th>\n",
       "      <th>answers_vector</th>\n",
       "      <th>documents_vector</th>\n",
       "      <th>answer_start_indexs</th>\n",
       "      <th>answer_end_indexs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[89, 93, 94, 2, 95, 1, 4, 18, 19, 88, 13, 16, ...</td>\n",
       "      <td>[31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...</td>\n",
       "      <td>56</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[8, 90, 97, 98, 2, 5, 21, 7, 88, 13, 16, 99, 1...</td>\n",
       "      <td>[34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[22, 89, 101, 9, 2, 6, 92, 13, 16, 102, 0, 0, ...</td>\n",
       "      <td>[38, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...</td>\n",
       "      <td>157</td>\n",
       "      <td>158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[103, 104, 8, 23, 24, 17, 2, 6, 92, 88, 13, 10...</td>\n",
       "      <td>[51, 23, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...</td>\n",
       "      <td>284</td>\n",
       "      <td>286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[90, 17, 2, 25, 26, 1, 13, 106, 6, 91, 0, 0, 0...</td>\n",
       "      <td>[71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...</td>\n",
       "      <td>535</td>\n",
       "      <td>536</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     question_vector  \\\n",
       "0  [89, 93, 94, 2, 95, 1, 4, 18, 19, 88, 13, 16, ...   \n",
       "1  [8, 90, 97, 98, 2, 5, 21, 7, 88, 13, 16, 99, 1...   \n",
       "2  [22, 89, 101, 9, 2, 6, 92, 13, 16, 102, 0, 0, ...   \n",
       "3  [103, 104, 8, 23, 24, 17, 2, 6, 92, 88, 13, 10...   \n",
       "4  [90, 17, 2, 25, 26, 1, 13, 106, 6, 91, 0, 0, 0...   \n",
       "\n",
       "                                      answers_vector  \\\n",
       "0  [31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...   \n",
       "1  [34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...   \n",
       "2  [38, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...   \n",
       "3  [51, 23, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "4  [71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...   \n",
       "\n",
       "                                    documents_vector  answer_start_indexs  \\\n",
       "0  [4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...                   56   \n",
       "1  [4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...                   73   \n",
       "2  [4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...                  157   \n",
       "3  [4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...                  284   \n",
       "4  [4, 18, 19, 9, 27, 8, 28, 29, 2, 30, 1, 31, 32...                  535   \n",
       "\n",
       "   answer_end_indexs  \n",
       "0                 57  \n",
       "1                 73  \n",
       "2                158  \n",
       "3                286  \n",
       "4                536  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorized_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "question_input = Input(shape=(80,), dtype='int32', name='question_input')\n",
    "context_input =  Input(shape=(1405,), dtype='int32', name='context_input')\n",
    "\n",
    "questionEmbd = Embedding(output_dim=100, input_dim=20000,\n",
    "                         mask_zero=False, \n",
    "                         input_length=80, trainable=False)(question_input)\n",
    "\n",
    "\n",
    "contextEmb = Embedding(output_dim=100, input_dim=20000,\n",
    "                         mask_zero=False,\n",
    "                         input_length=1405, trainable=False)(context_input)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q = Bidirectional(LSTM(80, return_sequences=True))(questionEmbd)\n",
    "D = Bidirectional(LSTM(40, return_sequences=True))(contextEmb)\n",
    "Q_flatten = Flatten()(Q)\n",
    "D_flatten = Flatten()(D)\n",
    "merged = concatenate([D_flatten, Q_flatten])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_4\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "context_input (InputLayer)      (None, 1405)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "question_input (InputLayer)     (None, 80)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_8 (Embedding)         (None, 1405, 100)    2000000     context_input[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "embedding_7 (Embedding)         (None, 80, 100)      2000000     question_input[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_6 (Bidirectional) (None, 1405, 80)     45120       embedding_8[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_5 (Bidirectional) (None, 80, 160)      115840      embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_6 (Flatten)             (None, 112400)       0           bidirectional_6[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 12800)        0           bidirectional_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)     (None, 125200)       0           flatten_6[0][0]                  \n",
      "                                                                 flatten_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 1)            125201      concatenate_6[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_7 (Concatenate)     (None, 125201)       0           concatenate_6[0][0]              \n",
      "                                                                 dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 1)            125202      concatenate_7[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 4,411,363\n",
      "Trainable params: 411,363\n",
      "Non-trainable params: 4,000,000\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rajatjain/.pyenv/versions/3.6.6/lib/python3.6/site-packages/ipykernel_launcher.py:5: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=[<tf.Tenso...)`\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "output1 = Dense(1,activation='sigmoid')(merged)\n",
    "l2_merged = concatenate([merged, output1])\n",
    "output2 = Dense(1,activation='sigmoid')(l2_merged)\n",
    "\n",
    "model = Model(inputs=[question_input,context_input], output = [output1,output2])\n",
    "model.compile(optimizer='adam', loss='mse', metrics=['mse'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions_padded = np.array(vectorized_data['question_vector'].values.tolist())\n",
    "documents_padded = np.array(vectorized_data['documents_vector'].values.tolist())\n",
    "answer_begin = np.array(vectorized_data['answers_vector'].values.tolist())\n",
    "answer_start_indexs = np.array(vectorized_data['answer_start_indexs'].values.tolist())\n",
    "answer_end_indexs = np.array(vectorized_data['answer_end_indexs'].values.tolist())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "2005/2005 [==============================] - 43s 22ms/step - loss: 415085.9400 - dense_5_loss: 206997.1935 - dense_6_loss: 208088.7495 - dense_5_mean_squared_error: 206997.1935 - dense_6_mean_squared_error: 208088.7495\n",
      "Epoch 2/10\n",
      "2005/2005 [==============================] - 46s 23ms/step - loss: 415085.7685 - dense_5_loss: 206997.1017 - dense_6_loss: 208088.6606 - dense_5_mean_squared_error: 206997.1017 - dense_6_mean_squared_error: 208088.6606\n",
      "Epoch 3/10\n",
      "2005/2005 [==============================] - 47s 23ms/step - loss: 415085.7692 - dense_5_loss: 206997.1049 - dense_6_loss: 208088.6603 - dense_5_mean_squared_error: 206997.1049 - dense_6_mean_squared_error: 208088.6603\n",
      "Epoch 4/10\n",
      "2005/2005 [==============================] - 42s 21ms/step - loss: 415085.7801 - dense_5_loss: 206997.1095 - dense_6_loss: 208088.6667 - dense_5_mean_squared_error: 206997.1095 - dense_6_mean_squared_error: 208088.6667\n",
      "Epoch 5/10\n",
      "2005/2005 [==============================] - 54s 27ms/step - loss: 415085.7642 - dense_5_loss: 206997.1026 - dense_6_loss: 208088.6600 - dense_5_mean_squared_error: 206997.1026 - dense_6_mean_squared_error: 208088.6600\n",
      "Epoch 6/10\n",
      "2005/2005 [==============================] - 40s 20ms/step - loss: 415085.7492 - dense_5_loss: 206997.0979 - dense_6_loss: 208088.6576 - dense_5_mean_squared_error: 206997.0979 - dense_6_mean_squared_error: 208088.6576\n",
      "Epoch 7/10\n",
      "2005/2005 [==============================] - 41s 20ms/step - loss: 415085.7572 - dense_5_loss: 206997.0991 - dense_6_loss: 208088.6580 - dense_5_mean_squared_error: 206997.0991 - dense_6_mean_squared_error: 208088.6580\n",
      "Epoch 8/10\n",
      "2005/2005 [==============================] - 45s 23ms/step - loss: 415085.7592 - dense_5_loss: 206997.1024 - dense_6_loss: 208088.6614 - dense_5_mean_squared_error: 206997.1024 - dense_6_mean_squared_error: 208088.6614\n",
      "Epoch 9/10\n",
      "2005/2005 [==============================] - 39s 19ms/step - loss: 415085.7495 - dense_5_loss: 206997.0906 - dense_6_loss: 208088.6589 - dense_5_mean_squared_error: 206997.0906 - dense_6_mean_squared_error: 208088.6589\n",
      "Epoch 10/10\n",
      "2005/2005 [==============================] - 44s 22ms/step - loss: 415085.7679 - dense_5_loss: 206997.1025 - dense_6_loss: 208088.6615 - dense_5_mean_squared_error: 206997.1025 - dense_6_mean_squared_error: 208088.6615\n"
     ]
    }
   ],
   "source": [
    "history = model.fit([questions_padded, documents_padded],[answer_start_indexs,  answer_end_indexs] ,\n",
    "                    epochs=10,\n",
    "                    batch_size=300)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
